home *** CD-ROM | disk | FTP | other *** search
/ Personal Computer World 2009 February / PCWFEB09.iso / Software / Resources / Chat & Communication / Digsby build 37 / digsby_setup.exe / lib / util / httplib2 / httplib2.pyo (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2008-10-13  |  37.8 KB  |  1,219 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyo (Python 2.5)
  3.  
  4. from __future__ import with_statement
  5. __author__ = 'Joe Gregorio (joe@bitworking.org)'
  6. __copyright__ = 'Copyright 2006, Joe Gregorio'
  7. __contributors__ = [
  8.     'Thomas Broyer (t.broyer@ltgt.net)',
  9.     'James Antill',
  10.     'Xavier Verges Farrero',
  11.     'Jonathan Feinberg',
  12.     'Blair Zajac',
  13.     'Sam Ruby',
  14.     'Louis Nyffenegger']
  15. __license__ = 'MIT'
  16. __version__ = 'digsby'
  17. import re
  18. import sys
  19. import email
  20. import email.Utils as email
  21. import email.Message as email
  22. import StringIO
  23. import gzip
  24. import zlib
  25. import httplib
  26. import urlparse
  27. import base64
  28. import os
  29. import copy
  30. import calendar
  31. import time
  32. import random
  33. import hmac
  34. from hashlib import md5, sha1
  35. from gettext import gettext as _
  36. import socket
  37. from logging import getLogger
  38. log = getLogger('httplib2')
  39.  
  40. try:
  41.     import socks
  42. except ImportError:
  43.     socks = None
  44.  
  45. if sys.version_info >= (2, 3):
  46.     from iri2uri import iri2uri
  47. else:
  48.     
  49.     def iri2uri(uri):
  50.         return uri
  51.  
  52. __all__ = [
  53.     'Http',
  54.     'Response',
  55.     'ProxyInfo',
  56.     'HttpLib2Error',
  57.     'RedirectMissingLocation',
  58.     'RedirectLimit',
  59.     'FailedToDecompressContent',
  60.     'UnimplementedDigestAuthOptionError',
  61.     'UnimplementedHmacDigestAuthOptionError',
  62.     'debuglevel',
  63.     'cached_content',
  64.     'set_default_proxy',
  65.     'get_default_proxy']
  66. debuglevel = 0
  67. if sys.version_info < (2, 4):
  68.     
  69.     def sorted(seq):
  70.         seq.sort()
  71.         return seq
  72.  
  73.  
  74.  
  75. def HTTPResponse__getheaders(self):
  76.     if self.msg is None:
  77.         raise httplib.ResponseNotReady()
  78.     
  79.     return self.msg.items()
  80.  
  81. if not hasattr(httplib.HTTPResponse, 'getheaders'):
  82.     httplib.HTTPResponse.getheaders = HTTPResponse__getheaders
  83.  
  84.  
  85. class HttpLib2Error(Exception):
  86.     pass
  87.  
  88.  
  89. class HttpLib2ErrorWithResponse(HttpLib2Error):
  90.     
  91.     def __init__(self, desc, response, content):
  92.         self.response = response
  93.         self.content = content
  94.         HttpLib2Error.__init__(self, desc)
  95.  
  96.  
  97.  
  98. class RedirectMissingLocation(HttpLib2ErrorWithResponse):
  99.     pass
  100.  
  101.  
  102. class RedirectLimit(HttpLib2ErrorWithResponse):
  103.     pass
  104.  
  105.  
  106. class FailedToDecompressContent(HttpLib2ErrorWithResponse):
  107.     pass
  108.  
  109.  
  110. class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse):
  111.     pass
  112.  
  113.  
  114. class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse):
  115.     pass
  116.  
  117.  
  118. class RelativeURIError(HttpLib2Error):
  119.     pass
  120.  
  121.  
  122. class ServerNotFoundError(HttpLib2Error):
  123.     pass
  124.  
  125. DEFAULT_MAX_REDIRECTS = 5
  126. HOP_BY_HOP = [
  127.     'connection',
  128.     'keep-alive',
  129.     'proxy-authenticate',
  130.     'proxy-authorization',
  131.     'te',
  132.     'trailers',
  133.     'transfer-encoding',
  134.     'upgrade']
  135.  
  136. def set_default_proxy(proxy_info):
  137.     ProxyInfo.default_proxy_info = proxy_info
  138.  
  139.  
  140. def get_default_proxy():
  141.     return ProxyInfo.default_proxy_info
  142.  
  143.  
  144. def _get_end2end_headers(response):
  145.     hopbyhop = list(HOP_BY_HOP)
  146.     []([ x.strip() for x in response.get('connection', '').split(',') ])
  147.     return _[2]
  148.  
  149. URI = re.compile('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?')
  150.  
  151. def parse_uri(uri):
  152.     groups = URI.match(uri).groups()
  153.     return (groups[1], groups[3], groups[4], groups[6], groups[8])
  154.  
  155.  
  156. def urlnorm(uri):
  157.     (scheme, authority, path, query, fragment) = parse_uri(uri)
  158.     if not scheme or not authority:
  159.         raise RelativeURIError('Only absolute URIs are allowed. uri = %s' % uri)
  160.     
  161.     authority = authority.lower()
  162.     scheme = scheme.lower()
  163.     if not path:
  164.         path = '/'
  165.     
  166.     if not query or '?'.join([
  167.         path,
  168.         query]):
  169.         pass
  170.     request_uri = path
  171.     scheme = scheme.lower()
  172.     defrag_uri = scheme + '://' + authority + request_uri
  173.     return (scheme, authority, request_uri, defrag_uri)
  174.  
  175. re_url_scheme = re.compile('^\\w+://')
  176. re_slash = re.compile('[?/:|]+')
  177.  
  178. def safename(filename):
  179.     
  180.     try:
  181.         if re_url_scheme.match(filename):
  182.             if isinstance(filename, str):
  183.                 filename = filename.decode('utf-8')
  184.                 filename = filename.encode('idna')
  185.             else:
  186.                 filename = filename.encode('idna')
  187.     except UnicodeError:
  188.         pass
  189.  
  190.     if isinstance(filename, unicode):
  191.         filename = filename.encode('utf-8')
  192.     
  193.     filemd5 = md5(filename).hexdigest()
  194.     filename = re_url_scheme.sub('', filename)
  195.     filename = re_slash.sub(',', filename)
  196.     if len(filename) > 200:
  197.         filename = filename[:200]
  198.     
  199.     return ','.join((filename, filemd5))
  200.  
  201. NORMALIZE_SPACE = re.compile('(?:\\r\\n)?[ \\t]+')
  202.  
  203. def _normalize_headers(headers):
  204.     return []([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for key, value in headers.iteritems() ])
  205.  
  206.  
  207. def _parse_cache_control(headers):
  208.     retval = { }
  209.     return retval
  210.  
  211. USE_WWW_AUTH_STRICT_PARSING = 0
  212. WWW_AUTH_STRICT = re.compile('^(?:\\s*(?:,\\s*)?([^\\0-\\x1f\\x7f-\\xff()<>@,;:\\\\\\"/[\\]?={} \\t]+)\\s*=\\s*\\"?((?<=\\")(?:[^\\0-\\x08\\x0A-\\x1f\\x7f-\\xff\\\\\\"]|\\\\[\\0-\\x7f])*?(?=\\")|(?<!\\")[^\\0-\\x1f\\x7f-\\xff()<>@,;:\\\\\\"/[\\]?={} \\t]+(?!\\"))\\"?)(.*)$')
  213. WWW_AUTH_RELAXED = re.compile('^(?:\\s*(?:,\\s*)?([^ \\t\\r\\n=]+)\\s*=\\s*\\"?((?<=\\")(?:[^\\\\\\"]|\\\\.)*?(?=\\")|(?<!\\")[^ \\t\\r\\n,]+(?!\\"))\\"?)(.*)$')
  214. UNQUOTE_PAIRS = re.compile('\\\\(.)')
  215.  
  216. def _parse_www_authenticate(headers, headername = 'www-authenticate'):
  217.     retval = { }
  218.     if headers.has_key(headername):
  219.         authenticate = headers[headername].strip()
  220.         if not USE_WWW_AUTH_STRICT_PARSING or WWW_AUTH_STRICT:
  221.             pass
  222.         www_auth = WWW_AUTH_RELAXED
  223.         while authenticate:
  224.             if headername == 'authentication-info':
  225.                 auth_scheme = 'digest'
  226.                 the_rest = authenticate
  227.             else:
  228.                 (auth_scheme, the_rest) = authenticate.split(' ', 1)
  229.             match = www_auth.search(the_rest)
  230.             auth_params = { }
  231.             while match:
  232.                 if match and len(match.groups()) == 3:
  233.                     (key, value, the_rest) = match.groups()
  234.                     auth_params[key.lower()] = UNQUOTE_PAIRS.sub('\\1', value)
  235.                 
  236.                 match = www_auth.search(the_rest)
  237.             retval[auth_scheme.lower()] = auth_params
  238.             authenticate = the_rest.strip()
  239.     
  240.     return retval
  241.  
  242.  
  243. def _entry_disposition(response_headers, request_headers):
  244.     retval = 'STALE'
  245.     cc = _parse_cache_control(request_headers)
  246.     cc_response = _parse_cache_control(response_headers)
  247.     if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1:
  248.         retval = 'TRANSPARENT'
  249.         if 'cache-control' not in request_headers:
  250.             request_headers['cache-control'] = 'no-cache'
  251.         
  252.     elif cc.has_key('no-cache'):
  253.         retval = 'TRANSPARENT'
  254.     elif cc_response.has_key('no-cache'):
  255.         retval = 'STALE'
  256.     elif cc.has_key('only-if-cached'):
  257.         retval = 'FRESH'
  258.     elif response_headers.has_key('date'):
  259.         date = calendar.timegm(email.Utils.parsedate_tz(response_headers['date']))
  260.         now = time.time()
  261.         current_age = max(0, now - date)
  262.         if cc_response.has_key('max-age'):
  263.             
  264.             try:
  265.                 freshness_lifetime = int(cc_response['max-age'])
  266.             except ValueError:
  267.                 freshness_lifetime = 0
  268.             except:
  269.                 None<EXCEPTION MATCH>ValueError
  270.             
  271.  
  272.         None<EXCEPTION MATCH>ValueError
  273.         if response_headers.has_key('expires'):
  274.             expires = email.Utils.parsedate_tz(response_headers['expires'])
  275.             if None == expires:
  276.                 freshness_lifetime = 0
  277.             else:
  278.                 freshness_lifetime = max(0, calendar.timegm(expires) - date)
  279.         else:
  280.             freshness_lifetime = 0
  281.         if cc.has_key('max-age'):
  282.             
  283.             try:
  284.                 freshness_lifetime = int(cc['max-age'])
  285.             except ValueError:
  286.                 freshness_lifetime = 0
  287.             except:
  288.                 None<EXCEPTION MATCH>ValueError
  289.             
  290.  
  291.         None<EXCEPTION MATCH>ValueError
  292.         if cc.has_key('min-fresh'):
  293.             
  294.             try:
  295.                 min_fresh = int(cc['min-fresh'])
  296.             except ValueError:
  297.                 min_fresh = 0
  298.  
  299.             current_age += min_fresh
  300.         
  301.         if freshness_lifetime > current_age:
  302.             retval = 'FRESH'
  303.         
  304.     
  305.     return retval
  306.  
  307.  
  308. def _decompressContent(response, new_content):
  309.     content = new_content
  310.     
  311.     try:
  312.         encoding = response.get('content-encoding', None)
  313.         if encoding in ('gzip', 'deflate'):
  314.             if encoding == 'gzip':
  315.                 content = gzip.GzipFile(fileobj = StringIO.StringIO(new_content)).read()
  316.             
  317.             if encoding == 'deflate':
  318.                 content = zlib.decompress(content)
  319.             
  320.             response['content-length'] = str(len(content))
  321.             response['-content-encoding'] = response['content-encoding']
  322.             del response['content-encoding']
  323.     except IOError:
  324.         content = ''
  325.         raise FailedToDecompressContent(_('Content purported to be compressed with %s but failed to decompress.') % response.get('content-encoding'), response, content)
  326.  
  327.     return content
  328.  
  329.  
  330. def _updateCache(request_headers, response_headers, content, cache, cachekey):
  331.     if cachekey:
  332.         cc = _parse_cache_control(request_headers)
  333.         cc_response = _parse_cache_control(response_headers)
  334.         if cc.has_key('no-store') or cc_response.has_key('no-store'):
  335.             cache.delete(cachekey)
  336.         else:
  337.             info = email.Message.Message()
  338.             for key, value in response_headers.iteritems():
  339.                 if key not in ('status', 'content-encoding', 'transfer-encoding'):
  340.                     info[key] = value
  341.                     continue
  342.             
  343.             status = response_headers.status
  344.             if status == 304:
  345.                 status = 200
  346.             
  347.             status_header = 'status: %d\r\n' % response_headers.status
  348.             header_str = info.as_string()
  349.             header_str = re.sub('\r(?!\n)|(?<!\r)\n', '\r\n', header_str)
  350.             text = ''.join([
  351.                 status_header,
  352.                 header_str,
  353.                 content])
  354.             cache.set(cachekey, text, content)
  355.     
  356.  
  357.  
  358. def _cnonce():
  359.     dig = time.ctime()([] % ([], [ '0123456789'[random.randrange(0, 9)] for i in range(20) ])).hexdigest()
  360.     return dig[:16]
  361.  
  362.  
  363. def _wsse_username_token(cnonce, iso_now, password):
  364.     return base64.encodestring(sha1('%s%s%s' % (cnonce, iso_now, password)).digest()).strip()
  365.  
  366.  
  367. class Authentication(object):
  368.     
  369.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  370.         (scheme, authority, path, query, fragment) = parse_uri(request_uri)
  371.         self.path = path
  372.         self.host = host
  373.         self.credentials = credentials
  374.         self.http = http
  375.  
  376.     
  377.     def depth(self, request_uri):
  378.         (scheme, authority, path, query, fragment) = parse_uri(request_uri)
  379.         return request_uri[len(self.path):].count('/')
  380.  
  381.     
  382.     def inscope(self, host, request_uri):
  383.         (scheme, authority, path, query, fragment) = parse_uri(request_uri)
  384.         if host == self.host:
  385.             pass
  386.         return path.startswith(self.path)
  387.  
  388.     
  389.     def request(self, method, request_uri, headers, content):
  390.         pass
  391.  
  392.     
  393.     def response(self, response, content):
  394.         return False
  395.  
  396.  
  397.  
  398. class BasicAuthentication(Authentication):
  399.     
  400.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  401.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  402.  
  403.     
  404.     def request(self, method, request_uri, headers, content):
  405.         headers['authorization'] = 'Basic ' + base64.encodestring('%s:%s' % self.credentials).strip()
  406.  
  407.  
  408.  
  409. class DigestAuthentication(Authentication):
  410.     
  411.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  412.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  413.         challenge = _parse_www_authenticate(response, 'www-authenticate')
  414.         self.challenge = challenge['digest']
  415.         qop = self.challenge.get('qop')
  416.         if not [] in [ x.strip() for x in qop.split() ] or 'auth':
  417.             pass
  418.         self.challenge['qop'] = None
  419.         self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5')
  420.         if self.challenge['algorithm'] != 'MD5':
  421.             raise UnimplementedDigestAuthOptionError(_('Unsupported value for algorithm: %s.' % self.challenge['algorithm']))
  422.         
  423.         self.A1 = ''.join([
  424.             self.credentials[0],
  425.             ':',
  426.             self.challenge['realm'],
  427.             ':',
  428.             self.credentials[1]])
  429.         self.challenge['nc'] = 1
  430.  
  431.     
  432.     def request(self, method, request_uri, headers, content, cnonce = None):
  433.         
  434.         H = lambda x: md5(x).hexdigest()
  435.         
  436.         KD = lambda s, d: H('%s:%s' % (s, d))
  437.         A2 = ''.join([
  438.             method,
  439.             ':',
  440.             request_uri])
  441.         if not cnonce:
  442.             pass
  443.         self.challenge['cnonce'] = _cnonce()
  444.         request_digest = '"%s"' % KD(H(self.A1), '%s:%s:%s:%s:%s' % (self.challenge['nonce'], '%08x' % self.challenge['nc'], self.challenge['cnonce'], self.challenge['qop'], H(A2)))
  445.         headers['Authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % (self.credentials[0], self.challenge['realm'], self.challenge['nonce'], request_uri, self.challenge['algorithm'], request_digest, self.challenge['qop'], self.challenge['nc'], self.challenge['cnonce'])
  446.         self.challenge['nc'] += 1
  447.  
  448.     
  449.     def response(self, response, content):
  450.         if not response.has_key('authentication-info'):
  451.             challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', { })
  452.             if 'true' == challenge.get('stale'):
  453.                 self.challenge['nonce'] = challenge['nonce']
  454.                 self.challenge['nc'] = 1
  455.                 return True
  456.             
  457.         else:
  458.             updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', { })
  459.             if updated_challenge.has_key('nextnonce'):
  460.                 self.challenge['nonce'] = updated_challenge['nextnonce']
  461.                 self.challenge['nc'] = 1
  462.             
  463.         return False
  464.  
  465.  
  466.  
  467. class HmacDigestAuthentication(Authentication):
  468.     __author__ = 'Thomas Broyer (t.broyer@ltgt.net)'
  469.     
  470.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  471.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  472.         challenge = _parse_www_authenticate(response, 'www-authenticate')
  473.         self.challenge = challenge['hmacdigest']
  474.         self.challenge['reason'] = self.challenge.get('reason', 'unauthorized')
  475.         if self.challenge['reason'] not in ('unauthorized', 'integrity'):
  476.             self.challenge['reason'] = 'unauthorized'
  477.         
  478.         self.challenge['salt'] = self.challenge.get('salt', '')
  479.         if not self.challenge.get('snonce'):
  480.             raise UnimplementedHmacDigestAuthOptionError(_("The challenge doesn't contain a server nonce, or this one is empty."))
  481.         
  482.         self.challenge['algorithm'] = self.challenge.get('algorithm', 'HMAC-SHA-1')
  483.         if self.challenge['algorithm'] not in ('HMAC-SHA-1', 'HMAC-MD5'):
  484.             raise UnimplementedHmacDigestAuthOptionError(_('Unsupported value for algorithm: %s.' % self.challenge['algorithm']))
  485.         
  486.         self.challenge['pw-algorithm'] = self.challenge.get('pw-algorithm', 'SHA-1')
  487.         if self.challenge['pw-algorithm'] not in ('SHA-1', 'MD5'):
  488.             raise UnimplementedHmacDigestAuthOptionError(_('Unsupported value for pw-algorithm: %s.' % self.challenge['pw-algorithm']))
  489.         
  490.         if self.challenge['algorithm'] == 'HMAC-MD5':
  491.             self.hashmod = md5
  492.         else:
  493.             self.hashmod = sha1
  494.         if self.challenge['pw-algorithm'] == 'MD5':
  495.             self.pwhashmod = md5
  496.         else:
  497.             self.pwhashmod = sha1
  498.         self.key = ''.join([
  499.             self.credentials[0],
  500.             ':',
  501.             self.pwhashmod(''.join([
  502.                 self.credentials[1],
  503.                 self.challenge['salt']])).hexdigest().lower(),
  504.             ':',
  505.             self.challenge['realm']])
  506.         self.key = self.pwhashmod(self.key).hexdigest().lower()
  507.  
  508.     
  509.     def request(self, method, request_uri, headers, content):
  510.         keys = _get_end2end_headers(headers)
  511.         keylist = []([ '%s ' % k for k in keys ])
  512.         headers_val = []([ headers[k] for k in keys ])
  513.         created = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
  514.         cnonce = _cnonce()
  515.         request_digest = '%s:%s:%s:%s:%s' % (method, request_uri, cnonce, self.challenge['snonce'], headers_val)
  516.         request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
  517.         headers['Authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % (self.credentials[0], self.challenge['realm'], self.challenge['snonce'], cnonce, request_uri, created, request_digest, keylist)
  518.  
  519.     
  520.     def response(self, response, content):
  521.         challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', { })
  522.         if challenge.get('reason') in ('integrity', 'stale'):
  523.             return True
  524.         
  525.         return False
  526.  
  527.  
  528.  
  529. class WsseAuthentication(Authentication):
  530.     
  531.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  532.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  533.  
  534.     
  535.     def request(self, method, request_uri, headers, content):
  536.         headers['Authorization'] = 'WSSE profile="UsernameToken"'
  537.         iso_now = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
  538.         cnonce = _cnonce()
  539.         password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
  540.         headers['X-WSSE'] = 'UsernameToken Username="%s", PasswordDigest="%s", Nonce="%s", Created="%s"' % (self.credentials[0], password_digest, cnonce, iso_now)
  541.  
  542.  
  543.  
  544. class GoogleLoginAuthentication(Authentication):
  545.     
  546.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  547.         urlencode = urlencode
  548.         import urllib
  549.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  550.         challenge = _parse_www_authenticate(response, 'www-authenticate')
  551.         service = challenge['googlelogin'].get('service', 'xapi')
  552.         if service == 'xapi' and request_uri.find('calendar') > 0:
  553.             service = 'cl'
  554.         
  555.         auth = dict(Email = credentials[0], Passwd = credentials[1], service = service, source = headers['user-agent'])
  556.         (resp, content) = self.http.request('https://www.google.com/accounts/ClientLogin', method = 'POST', body = urlencode(auth), headers = {
  557.             'Content-Type': 'application/x-www-form-urlencoded' })
  558.         lines = content.split('\n')
  559.         d = [](_[1])
  560.  
  561.     
  562.     def request(self, method, request_uri, headers, content):
  563.         headers['authorization'] = 'GoogleLogin Auth=' + self.Auth
  564.  
  565.  
  566. AUTH_SCHEME_CLASSES = {
  567.     'basic': BasicAuthentication,
  568.     'wsse': WsseAuthentication,
  569.     'digest': DigestAuthentication,
  570.     'hmacdigest': HmacDigestAuthentication,
  571.     'googlelogin': GoogleLoginAuthentication }
  572. AUTH_SCHEME_ORDER = [
  573.     'hmacdigest',
  574.     'googlelogin',
  575.     'digest',
  576.     'wsse',
  577.     'basic']
  578. from path import path
  579. from threading import RLock
  580.  
  581. class ThreadsafeFileCache(object):
  582.     
  583.     def __init__(self, cache):
  584.         self.cache = path(cache)
  585.         self.lock = RLock()
  586.         if not self.cache.exists():
  587.             self.cache.makedirs()
  588.         
  589.  
  590.     
  591.     def get(self, key):
  592.         self.lock.__enter__()
  593.         
  594.         try:
  595.             return (self.cache / safename(key)).bytes()
  596.         except IOError:
  597.             self.lock
  598.             self.lock
  599.         except:
  600.             self.lock
  601.         finally:
  602.             pass
  603.  
  604.  
  605.     
  606.     def getpath(self, key):
  607.         return self.cache / (safename(key) + '.dat')
  608.  
  609.     
  610.     def set(self, key, value, content):
  611.         filepath = self.cache / safename(key)
  612.         self.lock.__enter__()
  613.         
  614.         try:
  615.             filepath.write_bytes(value)
  616.             self.getpath(key).write_bytes(content)
  617.         finally:
  618.             pass
  619.  
  620.  
  621.     
  622.     def delete(self, key):
  623.         filepath = self.cache / safename(key)
  624.         self.lock.__enter__()
  625.         
  626.         try:
  627.             if filepath.exists():
  628.                 filepath.remove()
  629.             
  630.             if contentpath.exists():
  631.                 contentpath.remove()
  632.         finally:
  633.             pass
  634.  
  635.  
  636.     
  637.     def __repr__(self):
  638.         return '<ThreadsafeFileCache %s>' % self.cache
  639.  
  640.  
  641.  
  642. class Credentials(object):
  643.     
  644.     def __init__(self):
  645.         self.credentials = []
  646.  
  647.     
  648.     def add(self, name, password, domain = ''):
  649.         self.credentials.append((domain.lower(), name, password))
  650.  
  651.     
  652.     def clear(self):
  653.         self.credentials = []
  654.  
  655.     
  656.     def iter(self, domain):
  657.         for cdomain, name, password in self.credentials:
  658.             if cdomain == '' or domain == cdomain:
  659.                 yield (name, password)
  660.                 continue
  661.         
  662.  
  663.  
  664.  
  665. class KeyCerts(Credentials):
  666.     pass
  667.  
  668.  
  669. class ProxyInfo(object):
  670.     
  671.     def get_default_proxy():
  672.         return ProxyInfo.default_proxy_info
  673.  
  674.     get_default_proxy = staticmethod(get_default_proxy)
  675.     default_proxy_info = None
  676.     
  677.     def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns = None, proxy_user = None, proxy_pass = None):
  678.         (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass) = (proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
  679.  
  680.     
  681.     def astuple(self):
  682.         return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass)
  683.  
  684.     
  685.     def isgood(self):
  686.         if socks and self.proxy_host != None:
  687.             pass
  688.         return self.proxy_port != None
  689.  
  690.  
  691.  
  692. class HTTPConnectionWithTimeout(httplib.HTTPConnection):
  693.     
  694.     def __init__(self, host, port = None, strict = None, timeout = None, proxy_info = None):
  695.         httplib.HTTPConnection.__init__(self, host, port, strict)
  696.         self.timeout = timeout
  697.         if not proxy_info:
  698.             pass
  699.         self.proxy_info = ProxyInfo.get_default_proxy()
  700.  
  701.     
  702.     def connect(self):
  703.         msg = 'getaddrinfo returns an empty list'
  704.         for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
  705.             (af, socktype, proto, canonname, sa) = res
  706.             
  707.             try:
  708.                 if self.proxy_info and self.proxy_info.isgood():
  709.                     if self.proxy_info.proxy_type == socks.PROXY_TYPE_HTTP:
  710.                         self.sock = socket.socket(af, socktype, proto)
  711.                         self._orig_host = self.host
  712.                         self._orig_port = self.port
  713.                         self.host = self.proxy_info.proxy_host
  714.                         self.port = self.proxy_info.proxy_port
  715.                     else:
  716.                         self.sock = socks.socksocket(af, socktype, proto)
  717.                         self.sock.setproxy(*self.proxy_info.astuple())
  718.                 else:
  719.                     self.sock = socket.socket(af, socktype, proto)
  720.                 if self.timeout is not None:
  721.                     self.sock.settimeout(self.timeout)
  722.                 
  723.                 if self.debuglevel > 0:
  724.                     print 'connect: (%s, %s)' % (self.host, self.port)
  725.                 
  726.                 self.sock.connect(sa)
  727.             except socket.error:
  728.                 msg = None
  729.                 if self.debuglevel > 0:
  730.                     print 'connect fail:', (self.host, self.port), msg
  731.                 
  732.                 if self.sock:
  733.                     self.sock.close()
  734.                 
  735.                 self.sock = None
  736.                 continue
  737.  
  738.         
  739.         if not self.sock:
  740.             raise socket.error, msg
  741.         
  742.  
  743.  
  744. if hasattr(httplib, 'HTTPSConnection'):
  745.     
  746.     class HTTPSConnectionWithTimeout(httplib.HTTPSConnection):
  747.         
  748.         def __init__(self, host, port = None, key_file = None, cert_file = None, strict = None, timeout = None, proxy_info = None):
  749.             self.timeout = timeout
  750.             self.proxy_info = proxy_info
  751.             httplib.HTTPSConnection.__init__(self, host, port = port, key_file = key_file, cert_file = cert_file, strict = strict)
  752.  
  753.         
  754.         def connect(self):
  755.             if self.proxy_info and self.proxy_info.isgood():
  756.                 sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
  757.                 sock.setproxy(*self.proxy_info.astuple())
  758.             else:
  759.                 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  760.             if self.timeout is not None:
  761.                 sock.settimeout(self.timeout)
  762.             
  763.             sock.connect((self.host, self.port))
  764.             ssl = socket.ssl(sock, self.key_file, self.cert_file)
  765.             self.sock = httplib.FakeSocket(sock, ssl)
  766.  
  767.  
  768.  
  769. redirect_responses = set([
  770.     300,
  771.     301,
  772.     302,
  773.     303,
  774.     307])
  775.  
  776. class Http(object):
  777.     
  778.     def __init__(self, cache = None, timeout = None, proxy_info = None):
  779.         if not proxy_info:
  780.             pass
  781.         self.proxy_info = ProxyInfo.get_default_proxy()
  782.         self.connections = { }
  783.         if cache and isinstance(cache, basestring):
  784.             self.cache = ThreadsafeFileCache(cache)
  785.         else:
  786.             self.cache = cache
  787.         self.credentials = Credentials()
  788.         self.certificates = KeyCerts()
  789.         self.authorizations = []
  790.         self.follow_redirects = True
  791.         self.follow_all_redirects = False
  792.         self.ignore_etag = False
  793.         self.force_exception_to_status_code = False
  794.         self.timeout = timeout
  795.  
  796.     
  797.     def _auth_from_challenge(self, host, request_uri, headers, response, content):
  798.         challenges = _parse_www_authenticate(response, 'www-authenticate')
  799.         for cred in self.credentials.iter(host):
  800.             for scheme in AUTH_SCHEME_ORDER:
  801.                 if challenges.has_key(scheme):
  802.                     yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self)
  803.                     continue
  804.             
  805.         
  806.  
  807.     
  808.     def add_credentials(self, name, password, domain = ''):
  809.         self.credentials.add(name, password, domain)
  810.  
  811.     
  812.     def add_certificate(self, key, cert, domain):
  813.         self.certificates.add(key, cert, domain)
  814.  
  815.     
  816.     def clear_credentials(self):
  817.         self.credentials.clear()
  818.         self.authorizations = []
  819.  
  820.     
  821.     def _conn_request(self, conn, request_uri, method, body, headers):
  822.         for i in range(2):
  823.             
  824.             try:
  825.                 conn.request(method, request_uri, body, headers)
  826.                 response = conn.getresponse()
  827.             except socket.gaierror:
  828.                 conn.close()
  829.                 raise ServerNotFoundError('Unable to find the server at %s' % conn.host)
  830.             except (socket.error, httplib.HTTPException):
  831.                 if i == 0:
  832.                     conn.close()
  833.                     conn.connect()
  834.                     continue
  835.                 else:
  836.                     raise 
  837.             except:
  838.                 i == 0
  839.  
  840.             content = response.read()
  841.             response = Response(response)
  842.             if method != 'HEAD':
  843.                 content = _decompressContent(response, content)
  844.             
  845.         
  846.         return (response, content)
  847.  
  848.     
  849.     def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey):
  850.         auths = _[1]
  851.         if not auths or sorted(auths)[0][1]:
  852.             pass
  853.         auth = None
  854.         (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  855.         if auth:
  856.             if auth.response(response, body):
  857.                 auth.request(method, request_uri, headers, body)
  858.                 (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  859.                 response._stale_digest = 1
  860.             
  861.         
  862.         if response.status == 401:
  863.             for authorization in self._auth_from_challenge(host, request_uri, headers, response, content):
  864.                 authorization.request(method, request_uri, headers, body)
  865.                 (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  866.                 if response.status != 401:
  867.                     self.authorizations.append(authorization)
  868.                     authorization.response(response, body)
  869.                     break
  870.                     continue
  871.             
  872.         
  873.         if self.follow_all_redirects and method in ('GET', 'HEAD') or response.status == 303:
  874.             if self.follow_redirects and response.status in redirect_responses:
  875.                 if redirections:
  876.                     if not response.has_key('location') and response.status != 300:
  877.                         raise RedirectMissingLocation(_('Redirected but the response is missing a Location: header.'), response, content)
  878.                     
  879.                     if response.has_key('location'):
  880.                         location = response['location']
  881.                         (scheme, authority, path, query, fragment) = parse_uri(location)
  882.                         if authority == None:
  883.                             response['location'] = urlparse.urljoin(absolute_uri, location)
  884.                         
  885.                     
  886.                     if response.status == 301 and method in ('GET', 'HEAD'):
  887.                         response['-x-permanent-redirect-url'] = response['location']
  888.                         if not response.has_key('content-location'):
  889.                             response['content-location'] = absolute_uri
  890.                         
  891.                         _updateCache(headers, response, content, self.cache, cachekey)
  892.                         if cachekey:
  893.                             response.cache_path = self.cache.getpath(cachekey)
  894.                         
  895.                     
  896.                     if headers.has_key('if-none-match'):
  897.                         del headers['if-none-match']
  898.                     
  899.                     if headers.has_key('if-modified-since'):
  900.                         del headers['if-modified-since']
  901.                     
  902.                     if response.has_key('location'):
  903.                         location = response['location']
  904.                         old_response = copy.deepcopy(response)
  905.                         if not old_response.has_key('content-location'):
  906.                             old_response['content-location'] = absolute_uri
  907.                         
  908.                         if not response.status == 303 or method not in ('GET', 'HEAD') or 'GET':
  909.                             pass
  910.                         redirect_method = method
  911.                         (response, content) = self.request(location, redirect_method, body = body, headers = headers, redirections = redirections - 1)
  912.                         response.previous = old_response
  913.                     
  914.                 else:
  915.                     raise RedirectLimit(_('Redirected more times than rediection_limit allows.'), response, content)
  916.             elif response.status in (200, 203) and method == 'GET':
  917.                 if not response.has_key('content-location'):
  918.                     response['content-location'] = absolute_uri
  919.                 
  920.                 _updateCache(headers, response, content, self.cache, cachekey)
  921.                 if cachekey:
  922.                     response.cache_path = self.cache.getpath(cachekey)
  923.                 
  924.             
  925.         
  926.         return (response, content)
  927.  
  928.     
  929.     def request(self, uri, method = 'GET', body = None, headers = None, redirections = DEFAULT_MAX_REDIRECTS, connection_type = None, decode = True):
  930.         
  931.         try:
  932.             if headers is None:
  933.                 headers = { }
  934.             else:
  935.                 headers = _normalize_headers(headers)
  936.             if not headers.has_key('user-agent'):
  937.                 headers['user-agent'] = 'Python-httplib2/%s' % __version__
  938.             
  939.             uri = iri2uri(uri)
  940.             (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
  941.             domain_port = authority.split(':')[0:2]
  942.             if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http':
  943.                 scheme = 'https'
  944.                 authority = domain_port[0]
  945.             
  946.             if scheme == 'http' and self.proxy_info and self.proxy_info.isgood() and self.proxy_info.proxy_type == socks.PROXY_TYPE_HTTP:
  947.                 if not request_uri:
  948.                     slash = '/'
  949.                 else:
  950.                     slash = ''
  951.                 request_uri = uri + slash
  952.                 user = self.proxy_info.proxy_user
  953.                 password = self.proxy_info.proxy_pass
  954.                 if user and password:
  955.                     unquote = unquote
  956.                     import urllib
  957.                     user_pass = '%s:%s' % (unquote(user), unquote(password))
  958.                     creds = base64.b64encode(user_pass).strip()
  959.                     headers['Proxy-Authorization'] = 'Basic ' + creds
  960.                 
  961.                 newhost = self.proxy_info.proxy_host
  962.                 newport = self.proxy_info.proxy_port
  963.                 use_port = newport != httplib.HTTP_PORT
  964.                 if use_port:
  965.                     authority = newhost + ':' + str(newport)
  966.                 else:
  967.                     authority = newhost
  968.             
  969.             conn_key = scheme + ':' + authority
  970.             if conn_key in self.connections:
  971.                 conn = self.connections[conn_key]
  972.             elif not connection_type:
  973.                 if not scheme == 'https' or HTTPSConnectionWithTimeout:
  974.                     pass
  975.                 connection_type = HTTPConnectionWithTimeout
  976.             
  977.             certs = list(self.certificates.iter(authority))
  978.             if scheme == 'https' and certs:
  979.                 conn = self.connections[conn_key] = connection_type(authority, key_file = certs[0][0], cert_file = certs[0][1], timeout = self.timeout, proxy_info = self.proxy_info)
  980.             else:
  981.                 conn = self.connections[conn_key] = connection_type(authority, timeout = self.timeout, proxy_info = self.proxy_info)
  982.             conn.set_debuglevel(debuglevel)
  983.             if method in ('GET', 'HEAD') and 'range' not in headers:
  984.                 headers['accept-encoding'] = 'compress, gzip'
  985.             
  986.             info = email.Message.Message()
  987.             cached_value = None
  988.             if self.cache:
  989.                 cachekey = defrag_uri
  990.                 cached_value = self.cache.get(cachekey)
  991.                 if cached_value:
  992.                     info = email.message_from_string(cached_value)
  993.                     cache_path = self.cache.getpath(cachekey)
  994.                     
  995.                     try:
  996.                         content = cached_value.split('\r\n\r\n', 1)[1]
  997.                     except IndexError:
  998.                         self.cache.delete(cachekey)
  999.                         cachekey = None
  1000.                         cached_value = None
  1001.                         cache_path = None
  1002.                     except:
  1003.                         None<EXCEPTION MATCH>IndexError
  1004.                     
  1005.  
  1006.                 None<EXCEPTION MATCH>IndexError
  1007.             else:
  1008.                 cachekey = None
  1009.             if method in ('PUT',) and self.cache and info.has_key('etag') and not (self.ignore_etag) and 'if-match' not in headers:
  1010.                 headers['if-match'] = info['etag']
  1011.             
  1012.             if method not in ('GET', 'HEAD') and self.cache and cachekey:
  1013.                 self.cache.delete(cachekey)
  1014.             
  1015.             if cached_value and method in ('GET', 'HEAD') and self.cache and 'range' not in headers:
  1016.                 if info.has_key('-x-permanent-redirect-url'):
  1017.                     (response, new_content) = self.request(info['-x-permanent-redirect-url'], 'GET', headers = headers, redirections = redirections - 1)
  1018.                     response.previous = Response(info)
  1019.                     response.previous.fromcache = True
  1020.                 else:
  1021.                     entry_disposition = _entry_disposition(info, headers)
  1022.                     if entry_disposition == 'FRESH':
  1023.                         if not cached_value:
  1024.                             info['status'] = '504'
  1025.                             content = ''
  1026.                         
  1027.                         response = Response(info)
  1028.                         if cached_value:
  1029.                             response.fromcache = True
  1030.                             response.cache_path = cache_path
  1031.                         
  1032.                         return (response, content)
  1033.                     
  1034.                     if entry_disposition == 'STALE':
  1035.                         if info.has_key('etag') and not (self.ignore_etag) and 'if-none-match' not in headers:
  1036.                             headers['if-none-match'] = info['etag']
  1037.                         
  1038.                         if info.has_key('last-modified') and 'last-modified' not in headers:
  1039.                             headers['if-modified-since'] = info['last-modified']
  1040.                         
  1041.                     elif entry_disposition == 'TRANSPARENT':
  1042.                         pass
  1043.                     
  1044.                     (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
  1045.                 if response.status == 304 and method == 'GET':
  1046.                     for key in _get_end2end_headers(response):
  1047.                         info[key] = response[key]
  1048.                     
  1049.                     merged_response = Response(info)
  1050.                     if hasattr(response, '_stale_digest'):
  1051.                         merged_response._stale_digest = response._stale_digest
  1052.                     
  1053.                     _updateCache(headers, merged_response, content, self.cache, cachekey)
  1054.                     response = merged_response
  1055.                     response.status = 200
  1056.                     response.fromcache = True
  1057.                     response.cache_path = cache_path
  1058.                 elif response.status == 200:
  1059.                     content = new_content
  1060.                 else:
  1061.                     self.cache.delete(cachekey)
  1062.                     content = new_content
  1063.             else:
  1064.                 (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
  1065.         except Exception:
  1066.             e = None
  1067.             if self.force_exception_to_status_code:
  1068.                 if isinstance(e, HttpLib2ErrorWithResponse):
  1069.                     response = e.response
  1070.                     content = e.content
  1071.                     response.status = 500
  1072.                     response.reason = str(e)
  1073.                 elif isinstance(e, socket.timeout):
  1074.                     content = 'Request Timeout'
  1075.                     response = Response({
  1076.                         'content-type': 'text/plain',
  1077.                         'status': '408',
  1078.                         'content-length': len(content) })
  1079.                     response.reason = 'Request Timeout'
  1080.                 else:
  1081.                     content = str(e)
  1082.                     response = Response({
  1083.                         'content-type': 'text/plain',
  1084.                         'status': '400',
  1085.                         'content-length': len(content) })
  1086.                     response.reason = 'Bad Request'
  1087.             else:
  1088.                 raise 
  1089.         except:
  1090.             self.force_exception_to_status_code
  1091.  
  1092.         if decode:
  1093.             ctype_data = response.get('content-type', None)
  1094.             if ctype_data is not None:
  1095.                 while ctype_data:
  1096.                     split = ctype_data.split(';', 1)
  1097.                     if len(split) == 1:
  1098.                         split = [
  1099.                             split[0],
  1100.                             '']
  1101.                     
  1102.                     (param, ctype_data) = split
  1103.                     param = param.strip()
  1104.                     if '=' in param:
  1105.                         (key, val) = param.split('=', 1)
  1106.                         if key.lower() == 'charset':
  1107.                             charset = val
  1108.                             
  1109.                             try:
  1110.                                 content = content.decode(charset)
  1111.                             except:
  1112.                                 pass
  1113.  
  1114.                             ctype_data = None
  1115.                             continue
  1116.                         
  1117.                     key.lower() == 'charset'
  1118.             
  1119.         
  1120.         return (response, content)
  1121.  
  1122.  
  1123. HTTP = Http
  1124.  
  1125. class Response(dict):
  1126.     fromcache = False
  1127.     version = 11
  1128.     status = 200
  1129.     reason = 'Ok'
  1130.     previous = None
  1131.     
  1132.     def __init__(self, info):
  1133.         if isinstance(info, httplib.HTTPResponse):
  1134.             for key, value in info.getheaders():
  1135.                 self[key] = value
  1136.             
  1137.             self.status = info.status
  1138.             self['status'] = str(self.status)
  1139.             self.reason = info.reason
  1140.             self.version = info.version
  1141.         elif isinstance(info, email.Message.Message):
  1142.             for key, value in info.items():
  1143.                 self[key] = value
  1144.             
  1145.             self.status = int(self['status'])
  1146.         else:
  1147.             for key, value in info.iteritems():
  1148.                 self[key] = value
  1149.             
  1150.             self.status = int(self.get('status', self.status))
  1151.  
  1152.     
  1153.     def ok(self):
  1154.         return self.status // 100 == 2
  1155.  
  1156.     ok = property(ok)
  1157.     
  1158.     def __getattr__(self, name):
  1159.         if name == 'dict':
  1160.             return self
  1161.         else:
  1162.             raise AttributeError, name
  1163.  
  1164.  
  1165.  
  1166. def cached_content(cache_path):
  1167.     log.info('httplib2.cached_content(%s)' % cache_path)
  1168.     
  1169.     try:
  1170.         
  1171.         try:
  1172.             f = _[2]
  1173.             cached_value = f.read()
  1174.         finally:
  1175.             pass
  1176.  
  1177.         log.info('read %d bytes of data: %r', len(cached_value), cached_value[:40])
  1178.     except Exception:
  1179.         e = None
  1180.         log.error('error loading cached data from %s', cache_path)
  1181.         return None
  1182.  
  1183.     
  1184.     try:
  1185.         data = cached_value.split('\r\n\r\n', 1)[1]
  1186.         log.info('returning %d bytes of data', len(data))
  1187.         return data
  1188.     except IndexError:
  1189.         log.error('index error, returning none')
  1190.         return None
  1191.  
  1192.  
  1193.  
  1194. def main():
  1195.     import wx
  1196.     import PIL
  1197.     import cStringIO
  1198.     import gui.toolbox as gui
  1199.     
  1200.     def show(data):
  1201.         PIL.Image.open(cStringIO.StringIO(data)).Show()
  1202.  
  1203.     a = wx.PySimpleApp()
  1204.     f = wx.Frame(None)
  1205.     f.Show()
  1206.     url = 'http://www.google.com/intl/en_ALL/images/logo.gif'
  1207.     (resp, data) = Http('c:\\test_cache').request(url)
  1208.     print resp
  1209.     if resp.status <= resp.status:
  1210.         pass
  1211.     elif resp.status < 300:
  1212.         show(data)
  1213.     
  1214.     a.MainLoop()
  1215.  
  1216. if __name__ == '__main__':
  1217.     main()
  1218.  
  1219.